﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
using System.Threading;
using System.Windows.Forms;
using System.Text.RegularExpressions;
using System.Net;
using System.IO;

namespace g1_winform
{
    public partial class Form1 : Form
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private string Tips{ get; set; }

        private string url = null;
        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            url = textBox1.Text;
            Tips  = "";
            if (isLegal(url))
            {
                label2.Text = "非法地址";
            }
            else
            {
                label2.Text = "开始爬取" + url;
                try
                { 
                urls.Add(url, false);
                Crawl();
                }catch (Exception ex)
                {
                    label3.Text = ex.Message;
                    return;
                }
            }
            listBox1.Items.Add("end crawling!");
            label3.Text = Tips;
        }
        //public void StartCrawl()
        //{
        //    new Thread(this.Crawl).Start();
        //}
        private bool isLegal(string html)
        {
            try
            {

            
            string strRef = @"^(https?|ftp|file|ws)://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?$";
            Regex regex = new Regex(strRef);

            return Regex.IsMatch(strRef, html);
            }
            catch (Exception ex)
            {
                listBox1.Items.Add(ex.Message);
                return false;
            }
        }

        private void Crawl()
        {
            listBox1.Items.Add(" begin crawling\n");
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 10) break;
                listBox1.Items.Add("Crawling " + current + " page!\n");

                string html = Download(current);
                urls[current] = true;
                count++;
                if (isHtml(html))
                    Prase(html,current);
            }
        }

        public string Download(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                listBox1.Items.Add( ex.Message);
                return "";
            }
        }

        public void Prase(string html,string current)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', '>');
                if (strRef.StartsWith("/"))
                {
                    strRef = current + strRef;
                }
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }

        public bool isHtml(string html)
        {
            return new Regex("<html>").IsMatch(html);
        }

        private void listBox1_SelectedIndexChanged(object sender, EventArgs e)
        {

        }
    }
}
